{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>Language Model</th>\n",
       "      <th colspan=\"6\" halign=\"left\">google/canine-s</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pretrained</th>\n",
       "      <th colspan=\"3\" halign=\"left\">not_pretrained</th>\n",
       "      <th colspan=\"3\" halign=\"left\">pretrained</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Encoding</th>\n",
       "      <th>2-planar-brackets-greedy</th>\n",
       "      <th>arc-hybrid</th>\n",
       "      <th>relative</th>\n",
       "      <th>2-planar-brackets-greedy</th>\n",
       "      <th>arc-hybrid</th>\n",
       "      <th>relative</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Treebank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UD_Ancient_Greek-Perseus</th>\n",
       "      <td>13.6</td>\n",
       "      <td>15.4</td>\n",
       "      <td>17.3</td>\n",
       "      <td>18.4</td>\n",
       "      <td>20.3</td>\n",
       "      <td>20.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Armenian-ArmTDP</th>\n",
       "      <td>14.8</td>\n",
       "      <td>13.8</td>\n",
       "      <td>18.9</td>\n",
       "      <td>19.9</td>\n",
       "      <td>15.2</td>\n",
       "      <td>21.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Basque-BDT</th>\n",
       "      <td>18.8</td>\n",
       "      <td>16.1</td>\n",
       "      <td>20.7</td>\n",
       "      <td>22.4</td>\n",
       "      <td>19.5</td>\n",
       "      <td>22.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Bhojpuri-BHTB</th>\n",
       "      <td>13.4</td>\n",
       "      <td>17.7</td>\n",
       "      <td>22.4</td>\n",
       "      <td>11.3</td>\n",
       "      <td>10.3</td>\n",
       "      <td>17.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Bulgarian-BTB</th>\n",
       "      <td>23.8</td>\n",
       "      <td>21.2</td>\n",
       "      <td>26.3</td>\n",
       "      <td>29.4</td>\n",
       "      <td>25.4</td>\n",
       "      <td>30.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Chinese-GSDSimp</th>\n",
       "      <td>13.6</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.4</td>\n",
       "      <td>24.9</td>\n",
       "      <td>23.5</td>\n",
       "      <td>27.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Classical_Chinese-Kyoto</th>\n",
       "      <td>33.5</td>\n",
       "      <td>31.3</td>\n",
       "      <td>30.1</td>\n",
       "      <td>43.0</td>\n",
       "      <td>39.2</td>\n",
       "      <td>40.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Gothic-PROIEL</th>\n",
       "      <td>22.3</td>\n",
       "      <td>23.1</td>\n",
       "      <td>23.6</td>\n",
       "      <td>22.3</td>\n",
       "      <td>22.2</td>\n",
       "      <td>24.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Guajajara-TuDeT</th>\n",
       "      <td>29.4</td>\n",
       "      <td>22.0</td>\n",
       "      <td>27.9</td>\n",
       "      <td>29.8</td>\n",
       "      <td>21.5</td>\n",
       "      <td>27.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Kiche-IU</th>\n",
       "      <td>47.8</td>\n",
       "      <td>43.3</td>\n",
       "      <td>43.1</td>\n",
       "      <td>43.1</td>\n",
       "      <td>40.7</td>\n",
       "      <td>40.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Ligurian-GLT</th>\n",
       "      <td>4.8</td>\n",
       "      <td>10.8</td>\n",
       "      <td>13.5</td>\n",
       "      <td>5.7</td>\n",
       "      <td>11.7</td>\n",
       "      <td>12.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Maltese-MUDT</th>\n",
       "      <td>22.4</td>\n",
       "      <td>21.0</td>\n",
       "      <td>24.4</td>\n",
       "      <td>22.8</td>\n",
       "      <td>20.5</td>\n",
       "      <td>25.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Naija-NSC</th>\n",
       "      <td>26.7</td>\n",
       "      <td>24.5</td>\n",
       "      <td>28.8</td>\n",
       "      <td>30.4</td>\n",
       "      <td>28.9</td>\n",
       "      <td>31.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Old_East_Slavic-TOROT</th>\n",
       "      <td>21.3</td>\n",
       "      <td>20.4</td>\n",
       "      <td>23.5</td>\n",
       "      <td>22.7</td>\n",
       "      <td>22.0</td>\n",
       "      <td>25.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Skolt_Sami-Giellagas</th>\n",
       "      <td>10.5</td>\n",
       "      <td>9.3</td>\n",
       "      <td>9.2</td>\n",
       "      <td>10.3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Turkish-BOUN</th>\n",
       "      <td>18.8</td>\n",
       "      <td>16.5</td>\n",
       "      <td>21.8</td>\n",
       "      <td>23.7</td>\n",
       "      <td>21.5</td>\n",
       "      <td>24.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Vietnamese-VTB</th>\n",
       "      <td>14.1</td>\n",
       "      <td>13.4</td>\n",
       "      <td>16.1</td>\n",
       "      <td>15.3</td>\n",
       "      <td>13.2</td>\n",
       "      <td>16.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Welsh-CCG</th>\n",
       "      <td>25.8</td>\n",
       "      <td>24.3</td>\n",
       "      <td>27.9</td>\n",
       "      <td>20.5</td>\n",
       "      <td>19.8</td>\n",
       "      <td>23.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Wolof-WTB</th>\n",
       "      <td>22.8</td>\n",
       "      <td>21.8</td>\n",
       "      <td>26.7</td>\n",
       "      <td>20.2</td>\n",
       "      <td>18.8</td>\n",
       "      <td>25.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average</th>\n",
       "      <td>21.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>23.0</td>\n",
       "      <td>21.2</td>\n",
       "      <td>24.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Language Model                      google/canine-s                      \\\n",
       "Pretrained                           not_pretrained                       \n",
       "Encoding                   2-planar-brackets-greedy arc-hybrid relative   \n",
       "Treebank                                                                  \n",
       "UD_Ancient_Greek-Perseus                       13.6       15.4     17.3   \n",
       "UD_Armenian-ArmTDP                             14.8       13.8     18.9   \n",
       "UD_Basque-BDT                                  18.8       16.1     20.7   \n",
       "UD_Bhojpuri-BHTB                               13.4       17.7     22.4   \n",
       "UD_Bulgarian-BTB                               23.8       21.2     26.3   \n",
       "UD_Chinese-GSDSimp                             13.6       15.0     20.4   \n",
       "UD_Classical_Chinese-Kyoto                     33.5       31.3     30.1   \n",
       "UD_Gothic-PROIEL                               22.3       23.1     23.6   \n",
       "UD_Guajajara-TuDeT                             29.4       22.0     27.9   \n",
       "UD_Kiche-IU                                    47.8       43.3     43.1   \n",
       "UD_Ligurian-GLT                                 4.8       10.8     13.5   \n",
       "UD_Maltese-MUDT                                22.4       21.0     24.4   \n",
       "UD_Naija-NSC                                   26.7       24.5     28.8   \n",
       "UD_Old_East_Slavic-TOROT                       21.3       20.4     23.5   \n",
       "UD_Skolt_Sami-Giellagas                        10.5        9.3      9.2   \n",
       "UD_Turkish-BOUN                                18.8       16.5     21.8   \n",
       "UD_Vietnamese-VTB                              14.1       13.4     16.1   \n",
       "UD_Welsh-CCG                                   25.8       24.3     27.9   \n",
       "UD_Wolof-WTB                                   22.8       21.8     26.7   \n",
       "Average                                        21.0       20.0     23.3   \n",
       "\n",
       "Language Model                                                           \n",
       "Pretrained                               pretrained                      \n",
       "Encoding                   2-planar-brackets-greedy arc-hybrid relative  \n",
       "Treebank                                                                 \n",
       "UD_Ancient_Greek-Perseus                       18.4       20.3     20.6  \n",
       "UD_Armenian-ArmTDP                             19.9       15.2     21.2  \n",
       "UD_Basque-BDT                                  22.4       19.5     22.9  \n",
       "UD_Bhojpuri-BHTB                               11.3       10.3     17.8  \n",
       "UD_Bulgarian-BTB                               29.4       25.4     30.5  \n",
       "UD_Chinese-GSDSimp                             24.9       23.5     27.3  \n",
       "UD_Classical_Chinese-Kyoto                     43.0       39.2     40.3  \n",
       "UD_Gothic-PROIEL                               22.3       22.2     24.8  \n",
       "UD_Guajajara-TuDeT                             29.8       21.5     27.9  \n",
       "UD_Kiche-IU                                    43.1       40.7     40.3  \n",
       "UD_Ligurian-GLT                                 5.7       11.7     12.5  \n",
       "UD_Maltese-MUDT                                22.8       20.5     25.8  \n",
       "UD_Naija-NSC                                   30.4       28.9     31.1  \n",
       "UD_Old_East_Slavic-TOROT                       22.7       22.0     25.2  \n",
       "UD_Skolt_Sami-Giellagas                        10.3        8.0      8.0  \n",
       "UD_Turkish-BOUN                                23.7       21.5     24.3  \n",
       "UD_Vietnamese-VTB                              15.3       13.2     16.7  \n",
       "UD_Welsh-CCG                                   20.5       19.8     23.9  \n",
       "UD_Wolof-WTB                                   20.2       18.8     25.2  \n",
       "Average                                        23.0       21.2     24.5  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = pd.read_csv('dep_scores.csv')\n",
    "df['LAS'] = df['LAS'].map(lambda x: round(x, 1))\n",
    "df = df[(df['Finetuned'] == 'not_finetuned') & (df['Language Model'] == 'google/canine-s')\n",
    "& (df['Encoding'] != 'rel-pos')]\n",
    "df = df.sort_values(by=['Language Model', 'Finetuned'], ascending=[True, True])\n",
    "#df['Error'] = df['LAS'].map(lambda x: 100-x)\n",
    "#df['Error reduction'] = df.groupby(['Treebank', 'Language Model'])['Error'].diff()\n",
    "# df = df.dropna()\n",
    "#df['Relative error reduction'] = round((df['Error reduction'] / df['Error']) * 100, 1)\n",
    "# order by language model\n",
    "#df = df.sort_values(by=['Language Model'], ascending=[False])\n",
    "df = df.groupby(['Treebank', 'Language Model', 'Finetuned', 'Pretrained', 'Encoding']).mean()\n",
    "df = pd.pivot_table(df, values='LAS', index=['Treebank'], columns=['Language Model', 'Pretrained', 'Encoding'])\n",
    "\n",
    "# add a row with the average\n",
    "df.loc['Average'] = df.mean()\n",
    "df.round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>lm</th>\n",
       "      <th>bert-base-multilingual-cased</th>\n",
       "      <th>google/canine-c</th>\n",
       "      <th>google/canine-s</th>\n",
       "      <th>xlm-roberta-base</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>finetuned</th>\n",
       "      <th>not_finetuned</th>\n",
       "      <th>not_finetuned</th>\n",
       "      <th>not_finetuned</th>\n",
       "      <th>not_finetuned</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>language</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>basque</th>\n",
       "      <td>32.5</td>\n",
       "      <td>36.2</td>\n",
       "      <td>36.2</td>\n",
       "      <td>33.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chinese</th>\n",
       "      <td>16.1</td>\n",
       "      <td>16.9</td>\n",
       "      <td>17.3</td>\n",
       "      <td>8.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>english</th>\n",
       "      <td>12.9</td>\n",
       "      <td>9.9</td>\n",
       "      <td>9.9</td>\n",
       "      <td>14.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>french</th>\n",
       "      <td>14.5</td>\n",
       "      <td>14.1</td>\n",
       "      <td>13.9</td>\n",
       "      <td>15.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>german</th>\n",
       "      <td>18.8</td>\n",
       "      <td>15.4</td>\n",
       "      <td>16.4</td>\n",
       "      <td>18.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hebrew</th>\n",
       "      <td>41.5</td>\n",
       "      <td>40.8</td>\n",
       "      <td>41.0</td>\n",
       "      <td>43.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hungarian</th>\n",
       "      <td>40.0</td>\n",
       "      <td>37.4</td>\n",
       "      <td>37.4</td>\n",
       "      <td>39.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>korean</th>\n",
       "      <td>33.2</td>\n",
       "      <td>33.8</td>\n",
       "      <td>33.8</td>\n",
       "      <td>32.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>polish</th>\n",
       "      <td>42.8</td>\n",
       "      <td>40.1</td>\n",
       "      <td>40.1</td>\n",
       "      <td>41.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swedish</th>\n",
       "      <td>29.8</td>\n",
       "      <td>25.5</td>\n",
       "      <td>25.5</td>\n",
       "      <td>30.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average</th>\n",
       "      <td>28.2</td>\n",
       "      <td>27.0</td>\n",
       "      <td>27.2</td>\n",
       "      <td>27.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "lm        bert-base-multilingual-cased google/canine-c google/canine-s  \\\n",
       "finetuned                not_finetuned   not_finetuned   not_finetuned   \n",
       "language                                                                 \n",
       "basque                            32.5            36.2            36.2   \n",
       "chinese                           16.1            16.9            17.3   \n",
       "english                           12.9             9.9             9.9   \n",
       "french                            14.5            14.1            13.9   \n",
       "german                            18.8            15.4            16.4   \n",
       "hebrew                            41.5            40.8            41.0   \n",
       "hungarian                         40.0            37.4            37.4   \n",
       "korean                            33.2            33.8            33.8   \n",
       "polish                            42.8            40.1            40.1   \n",
       "swedish                           29.8            25.5            25.5   \n",
       "Average                           28.2            27.0            27.2   \n",
       "\n",
       "lm        xlm-roberta-base  \n",
       "finetuned    not_finetuned  \n",
       "language                    \n",
       "basque                33.7  \n",
       "chinese                8.2  \n",
       "english               14.5  \n",
       "french                15.4  \n",
       "german                18.1  \n",
       "hebrew                43.3  \n",
       "hungarian             39.5  \n",
       "korean                32.9  \n",
       "polish                41.8  \n",
       "swedish               30.7  \n",
       "Average               27.8  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('const_scores.csv')\n",
    "df['BracketingFMeasure'] = df['BracketingFMeasure'].map(lambda x: round(x, 1))\n",
    "df = df[(df['finetuned'] == 'not_finetuned') & (df['pretrained'] == 'not_pretrained')]\n",
    "df = df.sort_values(by=['language', 'lm', 'finetuned'], ascending=[True, True, True])\n",
    "#df['Error'] = df['BracketingFMeasure'].map(lambda x: 100-x)\n",
    "#df['Error reduction'] = df.groupby(['language', 'lm'])['Error'].diff()\n",
    "\n",
    "#df['Relative error reduction'] = round((df['Error reduction'] / df['Error']) * 100, 1)\n",
    "# order by language model\n",
    "df = df.sort_values(by=['lm'], ascending=[False])\n",
    "#df = df.groupby(['language', 'lm', 'finetuned', 'pretrained']).mean()\n",
    "df = pd.pivot_table(df, values='BracketingFMeasure', index=['language'], columns=['lm','finetuned'])\n",
    "\n",
    "# add a row with the average\n",
    "df.loc['Average'] = df.mean()\n",
    "df = df.dropna(axis=1)\n",
    "df.round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mbert_languages = {\"Afrikaans\", \"Albanian\", \"Arabic\", \"Aragonese\", \"Armenian\", \"Asturian\", \"Azerbaijani\", \"Bashkir\", \"Basque\", \"Bavarian\", \"Belarusian\", \"Bengali\", \"Bishnupriya Manipuri\", \"Bosnian\", \"Breton\", \"Bulgarian\", \"Burmese\", \"Catalan\", \"Cebuano\", \"Chechen\", \"Chinese (Simplified)\", \"Chinese (Traditional)\", \"Chuvash\", \"Croatian\", \"Czech\", \"Danish\", \"Dutch\", \"English\", \"Estonian\", \"Finnish\", \"French\", \"Galician\", \"Georgian\", \"German\", \"Greek\", \"Gujarati\", \"Haitian\", \"Hebrew\", \"Hindi\", \"Hungarian\", \"Icelandic\", \"Ido\", \"Indonesian\", \"Irish\", \"Italian\", \"Japanese\", \"Javanese\", \"Kannada\", \"Kazakh\", \"Kirghiz\", \"Korean\", \"Latin\", \"Latvian\", \"Lithuanian\", \"Lombard\", \"Low Saxon\", \"Luxembourgish\", \"Macedonian\", \"Malagasy\", \"Malay\", \"Malayalam\", \"Marathi\", \"Minangkabau\", \"Nepali\", \"Newar\", \"Norwegian (Bokmal)\", \"Norwegian (Nynorsk)\", \"Occitan\", \"Persian (Farsi)\", \"Piedmontese\", \"Polish\", \"Portuguese\", \"Punjabi\", \"Romanian\", \"Russian\", \"Scots\", \"Serbian\", \"Serbo-Croatian\", \"Sicilian\", \"Slovak\", \"Slovenian\", \"South Azerbaijani\", \"Spanish\", \"Sundanese\", \"Swahili\", \"Swedish\", \"Tagalog\", \"Tajik\", \"Tamil\", \"Tatar\", \"Telugu\", \"Turkish\", \"Ukrainian\", \"Urdu\", \"Uzbek\", \"Vietnamese\", \"Volapük\", \"Waray-Waray\", \"Welsh\", \"West Frisian\", \"Western Punjabi\", \"Yoruba\"}\n",
    "xlm_roberta_languages = {\"Afrikaans\", \"Albanian\", \"Amharic\", \"Arabic\", \"Armenian\", \"Assamese\", \"Azerbaijani\", \"Basque\", \"Belarusian\", \"Bengali\", \"Bengali Romanize\", \"Bosnian\", \"Breton\", \"Bulgarian\", \"Burmese\", \"Burmese zawgyi font\", \"Catalan\", \"Chinese (Simplified)\", \"Chinese (Traditional)\", \"Croatian\", \"Czech\", \"Danish\", \"Dutch\", \"English\", \"Esperanto\", \"Estonian\", \"Filipino\", \"Finnish\", \"French\", \"Galician\", \"Georgian\", \"German\", \"Greek\", \"Gujarati\", \"Hausa\", \"Hebrew\", \"Hindi\", \"Hindi Romanize\", \"Hungarian\", \"Icelandic\", \"Indonesian\", \"Irish\", \"Italian\", \"Japanese\", \"Javanese\", \"Kannada\", \"Kazakh\", \"Khmer\", \"Korean\", \"Kurdish (Kurmanji)\", \"Kyrgyz\", \"Lao\", \"Latin\", \"Latvian\", \"Lithuanian\", \"Macedonian\", \"Malagasy\", \"Malay\", \"Malayalam\", \"Marathi\", \"Mongolian\", \"Nepali\", \"Norwegian\", \"Oriya\", \"Oromo\", \"Pashto\", \"Persian\", \"Polish\", \"Portuguese\", \"Punjabi\", \"Romanian\", \"Russian\", \"Sanskrit\", \"Scottish Gaelic\", \"Serbian\", \"Sindhi\", \"Sinhala\", \"Slovak\", \"Slovenian\", \"Somali\", \"Spanish\", \"Sundanese\", \"Swahili\", \"Swedish\", \"Tamil\", \"Tamil Romanize\", \"Telugu\", \"Telugu Romanize\", \"Thai\", \"Turkish\", \"Ukrainian\", \"Urdu\", \"Urdu Romanize\", \"Uyghur\", \"Uzbek\", \"Vietnamese\", \"Welsh\", \"Western Frisian\", \"Xhosa\", \"Yiddish\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# differences between mbert and xlm-roberta languages\n",
    "mbert_unique = mbert_languages - xlm_roberta_languages\n",
    "xlm_roberta_unique = xlm_roberta_languages - mbert_languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33, 31)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mbert_unique), len(xlm_roberta_unique)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'Aragonese',\n",
       "  'Asturian',\n",
       "  'Bashkir',\n",
       "  'Bavarian',\n",
       "  'Bishnupriya Manipuri',\n",
       "  'Cebuano',\n",
       "  'Chechen',\n",
       "  'Chuvash',\n",
       "  'Haitian',\n",
       "  'Ido',\n",
       "  'Kirghiz',\n",
       "  'Lombard',\n",
       "  'Low Saxon',\n",
       "  'Luxembourgish',\n",
       "  'Minangkabau',\n",
       "  'Newar',\n",
       "  'Norwegian (Bokmal)',\n",
       "  'Norwegian (Nynorsk)',\n",
       "  'Occitan',\n",
       "  'Persian (Farsi)',\n",
       "  'Piedmontese',\n",
       "  'Scots',\n",
       "  'Serbo-Croatian',\n",
       "  'Sicilian',\n",
       "  'South Azerbaijani',\n",
       "  'Tagalog',\n",
       "  'Tajik',\n",
       "  'Tatar',\n",
       "  'Volapük',\n",
       "  'Waray-Waray',\n",
       "  'West Frisian',\n",
       "  'Western Punjabi',\n",
       "  'Yoruba'},\n",
       " {'Amharic',\n",
       "  'Assamese',\n",
       "  'Bengali Romanize',\n",
       "  'Burmese zawgyi font',\n",
       "  'Esperanto',\n",
       "  'Filipino',\n",
       "  'Hausa',\n",
       "  'Hindi Romanize',\n",
       "  'Khmer',\n",
       "  'Kurdish (Kurmanji)',\n",
       "  'Kyrgyz',\n",
       "  'Lao',\n",
       "  'Mongolian',\n",
       "  'Norwegian',\n",
       "  'Oriya',\n",
       "  'Oromo',\n",
       "  'Pashto',\n",
       "  'Persian',\n",
       "  'Sanskrit',\n",
       "  'Scottish Gaelic',\n",
       "  'Sindhi',\n",
       "  'Sinhala',\n",
       "  'Somali',\n",
       "  'Tamil Romanize',\n",
       "  'Telugu Romanize',\n",
       "  'Thai',\n",
       "  'Urdu Romanize',\n",
       "  'Uyghur',\n",
       "  'Western Frisian',\n",
       "  'Xhosa',\n",
       "  'Yiddish'})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mbert_unique, xlm_roberta_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "def avg_displacement(conllu_file):\n",
    "    \"\"\"\n",
    "    Measures the average dependency displacement of a given conllu file\n",
    "    \"\"\"\n",
    "    with open(conllu_file, 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    \n",
    "    # get all the lines with dependency relations\n",
    "    dep_lines = [line for line in lines if line[0].isdigit()]\n",
    "\n",
    "    # get all the displacements\n",
    "    displacements = []\n",
    "    for line in dep_lines:\n",
    "        line = line.split('\\t')\n",
    "        try:\n",
    "            head = int(line[6])\n",
    "            dep = int(line[0])\n",
    "            displacements.append(head - dep)\n",
    "        except ValueError:\n",
    "            continue\n",
    "    \n",
    "    return sum(displacements) / len(displacements)\n",
    "\n",
    "\n",
    "treebanks = [\n",
    "    'UD_Ancient_Greek-Perseus', 'UD_Skolt_Sami-Giellagas', 'UD_Welsh-CCG',\n",
    "    'UD_Bulgarian-BTB', 'UD_Guajajara-TuDeT', 'UD_Armenian-ArmTDP',\n",
    "    'UD_Turkish-BOUN', 'UD_Ligurian-GLT', 'UD_Vietnamese-VTB',\n",
    "    'UD_Basque-BDT', 'UD_Bhojpuri-BHTB', 'UD_Kiche-IU', 'UD_Chinese-GSDSimp',\n",
    "        ]\n",
    "# define ud directory\n",
    "ud = '/home/alberto/Universal Dependencies 2.9/ud-treebanks-v2.9/'\n",
    "displacement_dic = {}\n",
    "for treebank in treebanks:\n",
    "    # get test conllu file\n",
    "    treebank_dir = ud + treebank + '/'\n",
    "    for file in os.listdir(treebank_dir):\n",
    "            if file.endswith('ud-test.conllu'):\n",
    "                    gold_conllu = treebank_dir + file\n",
    "                    displacement_dic[treebank] = round(avg_displacement(gold_conllu),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'UD_Ancient_Greek-Perseus': -0.54,\n",
       " 'UD_Skolt_Sami-Giellagas': -1.04,\n",
       " 'UD_Welsh-CCG': -1.41,\n",
       " 'UD_Bulgarian-BTB': -0.92,\n",
       " 'UD_Guajajara-TuDeT': -1.84,\n",
       " 'UD_Armenian-ArmTDP': -0.85,\n",
       " 'UD_Turkish-BOUN': 0.56,\n",
       " 'UD_Ligurian-GLT': -0.77,\n",
       " 'UD_Vietnamese-VTB': -1.29,\n",
       " 'UD_Basque-BDT': -0.62,\n",
       " 'UD_Bhojpuri-BHTB': 0.9,\n",
       " 'UD_Kiche-IU': -1.28,\n",
       " 'UD_Chinese-GSDSimp': -0.14}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "displacement_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Treebank</th>\n",
       "      <th>Language Model</th>\n",
       "      <th>Difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>basque</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>24.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>basque</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-3.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>basque</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>5.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>basque</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>13.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>chinese</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>40.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>chinese</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>8.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>chinese</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>21.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>chinese</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>37.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>english</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>44.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>english</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-0.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>english</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>7.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>english</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>26.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>french</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>35.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>french</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-1.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>french</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>6.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>french</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>16.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>german</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>26.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>german</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>3.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>german</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>7.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>german</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>23.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>hebrew</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>32.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>hebrew</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-11.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>hebrew</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>-0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>hebrew</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>16.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>hungarian</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>29.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>hungarian</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-5.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>hungarian</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>3.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>hungarian</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>26.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>korean</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>24.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>korean</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>3.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>korean</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>8.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>korean</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>20.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>polish</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>34.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>polish</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-6.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>polish</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>polish</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>26.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>swedish</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>26.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>swedish</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>-2.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>swedish</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>3.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>swedish</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>11.59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Treebank                Language Model  Difference\n",
       "5       basque  bert-base-multilingual-cased       24.13\n",
       "39      basque               google/canine-c       -3.13\n",
       "37      basque               google/canine-s        5.27\n",
       "7       basque              xlm-roberta-base       13.44\n",
       "73     chinese  bert-base-multilingual-cased       40.47\n",
       "77     chinese               google/canine-c        8.70\n",
       "75     chinese               google/canine-s       21.94\n",
       "120    chinese              xlm-roberta-base       37.40\n",
       "29     english  bert-base-multilingual-cased       44.36\n",
       "63     english               google/canine-c       -0.27\n",
       "61     english               google/canine-s        7.59\n",
       "31     english              xlm-roberta-base       26.06\n",
       "0       french  bert-base-multilingual-cased       35.57\n",
       "33      french               google/canine-c       -1.98\n",
       "35      french               google/canine-s        6.27\n",
       "3       french              xlm-roberta-base       16.98\n",
       "65      german  bert-base-multilingual-cased       26.64\n",
       "69      german               google/canine-c        3.08\n",
       "71      german               google/canine-s        7.61\n",
       "67      german              xlm-roberta-base       23.03\n",
       "9       hebrew  bert-base-multilingual-cased       32.93\n",
       "43      hebrew               google/canine-c      -11.01\n",
       "41      hebrew               google/canine-s       -0.81\n",
       "11      hebrew              xlm-roberta-base       16.64\n",
       "13   hungarian  bert-base-multilingual-cased       29.79\n",
       "47   hungarian               google/canine-c       -5.90\n",
       "45   hungarian               google/canine-s        3.60\n",
       "15   hungarian              xlm-roberta-base       26.44\n",
       "17      korean  bert-base-multilingual-cased       24.17\n",
       "51      korean               google/canine-c        3.67\n",
       "49      korean               google/canine-s        8.08\n",
       "19      korean              xlm-roberta-base       20.32\n",
       "21      polish  bert-base-multilingual-cased       34.21\n",
       "55      polish               google/canine-c       -6.20\n",
       "53      polish               google/canine-s        2.75\n",
       "23      polish              xlm-roberta-base       26.17\n",
       "25     swedish  bert-base-multilingual-cased       26.17\n",
       "59     swedish               google/canine-c       -2.80\n",
       "57     swedish               google/canine-s        3.24\n",
       "27     swedish              xlm-roberta-base       11.59"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Constituency\n",
    "df = pd.read_csv('const_scores.csv')\n",
    "df = df[df['finetuned'] == 'not_finetuned']\n",
    "df = df.drop_duplicates()\n",
    "df['Treebank'] = df['language']\n",
    "df['Language Model'] = df['lm']\n",
    "df['Pretrained'] = df['pretrained']\n",
    "df['F-Score'] = df['BracketingFMeasure']\n",
    "# drop old columns\n",
    "df = df.drop(columns=['Unnamed: 16', 'language', 'lm', 'pretrained', 'BracketingFMeasure'])\n",
    "df = df.sort_values(by=['Treebank', 'Language Model', 'Pretrained'])\n",
    "df[\"Difference\"] = df.groupby(['Treebank', 'Language Model'])['F-Score'].diff()\n",
    "df = df.dropna()\n",
    "df[['Treebank', 'Language Model', 'Difference']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>finetuned</th>\n",
       "      <th>encoding</th>\n",
       "      <th>task</th>\n",
       "      <th>Numberofsentence</th>\n",
       "      <th>NumberofErrorsentence</th>\n",
       "      <th>NumberofSkipsentence</th>\n",
       "      <th>NumberofValidsentence</th>\n",
       "      <th>BracketingRecall</th>\n",
       "      <th>BracketingPrecision</th>\n",
       "      <th>Completematch</th>\n",
       "      <th>Averagecrossing</th>\n",
       "      <th>Nocrossing</th>\n",
       "      <th>Treebank</th>\n",
       "      <th>Language Model</th>\n",
       "      <th>Pretrained</th>\n",
       "      <th>F-Score</th>\n",
       "      <th>Difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>946</td>\n",
       "      <td>51.29</td>\n",
       "      <td>63.08</td>\n",
       "      <td>2.33</td>\n",
       "      <td>1.33</td>\n",
       "      <td>46.19</td>\n",
       "      <td>basque</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>56.58</td>\n",
       "      <td>24.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>946</td>\n",
       "      <td>27.73</td>\n",
       "      <td>40.84</td>\n",
       "      <td>0.63</td>\n",
       "      <td>2.43</td>\n",
       "      <td>29.28</td>\n",
       "      <td>basque</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>33.03</td>\n",
       "      <td>-3.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>946</td>\n",
       "      <td>34.92</td>\n",
       "      <td>50.94</td>\n",
       "      <td>0.63</td>\n",
       "      <td>1.66</td>\n",
       "      <td>40.27</td>\n",
       "      <td>basque</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>41.43</td>\n",
       "      <td>5.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>946</td>\n",
       "      <td>36.10</td>\n",
       "      <td>67.83</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.63</td>\n",
       "      <td>68.92</td>\n",
       "      <td>basque</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>47.12</td>\n",
       "      <td>13.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>348</td>\n",
       "      <td>51.85</td>\n",
       "      <td>62.35</td>\n",
       "      <td>17.24</td>\n",
       "      <td>3.74</td>\n",
       "      <td>31.90</td>\n",
       "      <td>chinese</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>56.61</td>\n",
       "      <td>40.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>348</td>\n",
       "      <td>20.57</td>\n",
       "      <td>33.70</td>\n",
       "      <td>16.67</td>\n",
       "      <td>5.97</td>\n",
       "      <td>29.02</td>\n",
       "      <td>chinese</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>25.55</td>\n",
       "      <td>8.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>348</td>\n",
       "      <td>34.74</td>\n",
       "      <td>45.20</td>\n",
       "      <td>15.52</td>\n",
       "      <td>5.87</td>\n",
       "      <td>25.00</td>\n",
       "      <td>chinese</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>39.28</td>\n",
       "      <td>21.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>348</td>\n",
       "      <td>33.63</td>\n",
       "      <td>46.82</td>\n",
       "      <td>16.95</td>\n",
       "      <td>5.43</td>\n",
       "      <td>26.72</td>\n",
       "      <td>chinese</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>45.60</td>\n",
       "      <td>37.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2416</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2416</td>\n",
       "      <td>54.93</td>\n",
       "      <td>59.77</td>\n",
       "      <td>5.13</td>\n",
       "      <td>4.52</td>\n",
       "      <td>22.68</td>\n",
       "      <td>english</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>57.25</td>\n",
       "      <td>44.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2416</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2416</td>\n",
       "      <td>8.93</td>\n",
       "      <td>10.43</td>\n",
       "      <td>1.32</td>\n",
       "      <td>11.43</td>\n",
       "      <td>4.43</td>\n",
       "      <td>english</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>9.62</td>\n",
       "      <td>-0.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2416</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2416</td>\n",
       "      <td>16.54</td>\n",
       "      <td>18.52</td>\n",
       "      <td>0.75</td>\n",
       "      <td>10.40</td>\n",
       "      <td>5.50</td>\n",
       "      <td>english</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>17.48</td>\n",
       "      <td>7.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2416</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2416</td>\n",
       "      <td>38.27</td>\n",
       "      <td>43.06</td>\n",
       "      <td>2.19</td>\n",
       "      <td>7.14</td>\n",
       "      <td>9.19</td>\n",
       "      <td>english</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>40.52</td>\n",
       "      <td>26.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2541</td>\n",
       "      <td>49.30</td>\n",
       "      <td>50.85</td>\n",
       "      <td>3.74</td>\n",
       "      <td>6.28</td>\n",
       "      <td>20.19</td>\n",
       "      <td>french</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>50.06</td>\n",
       "      <td>35.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2541</td>\n",
       "      <td>12.22</td>\n",
       "      <td>12.01</td>\n",
       "      <td>1.50</td>\n",
       "      <td>14.93</td>\n",
       "      <td>9.64</td>\n",
       "      <td>french</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>12.11</td>\n",
       "      <td>-1.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2541</td>\n",
       "      <td>20.20</td>\n",
       "      <td>20.07</td>\n",
       "      <td>0.55</td>\n",
       "      <td>12.31</td>\n",
       "      <td>9.13</td>\n",
       "      <td>french</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>20.14</td>\n",
       "      <td>6.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2541</td>\n",
       "      <td>31.41</td>\n",
       "      <td>33.33</td>\n",
       "      <td>1.34</td>\n",
       "      <td>9.80</td>\n",
       "      <td>12.91</td>\n",
       "      <td>french</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>32.35</td>\n",
       "      <td>16.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>41.19</td>\n",
       "      <td>50.54</td>\n",
       "      <td>10.84</td>\n",
       "      <td>1.11</td>\n",
       "      <td>50.36</td>\n",
       "      <td>german</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>45.39</td>\n",
       "      <td>26.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>14.15</td>\n",
       "      <td>26.54</td>\n",
       "      <td>5.10</td>\n",
       "      <td>1.35</td>\n",
       "      <td>45.98</td>\n",
       "      <td>german</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>18.46</td>\n",
       "      <td>3.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>21.24</td>\n",
       "      <td>27.55</td>\n",
       "      <td>5.72</td>\n",
       "      <td>1.90</td>\n",
       "      <td>34.12</td>\n",
       "      <td>german</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>23.98</td>\n",
       "      <td>7.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>5000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>35.89</td>\n",
       "      <td>48.21</td>\n",
       "      <td>8.00</td>\n",
       "      <td>1.09</td>\n",
       "      <td>50.38</td>\n",
       "      <td>german</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>41.15</td>\n",
       "      <td>23.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>716</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>716</td>\n",
       "      <td>72.48</td>\n",
       "      <td>76.53</td>\n",
       "      <td>1.12</td>\n",
       "      <td>5.70</td>\n",
       "      <td>18.58</td>\n",
       "      <td>hebrew</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>74.45</td>\n",
       "      <td>32.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>716</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>716</td>\n",
       "      <td>28.71</td>\n",
       "      <td>31.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>13.78</td>\n",
       "      <td>4.19</td>\n",
       "      <td>hebrew</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>29.82</td>\n",
       "      <td>-11.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>716</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>716</td>\n",
       "      <td>38.46</td>\n",
       "      <td>41.99</td>\n",
       "      <td>0.14</td>\n",
       "      <td>10.78</td>\n",
       "      <td>6.15</td>\n",
       "      <td>hebrew</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>40.15</td>\n",
       "      <td>-0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>716</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>716</td>\n",
       "      <td>58.61</td>\n",
       "      <td>61.32</td>\n",
       "      <td>0.00</td>\n",
       "      <td>10.50</td>\n",
       "      <td>5.45</td>\n",
       "      <td>hebrew</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>59.93</td>\n",
       "      <td>16.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1009</td>\n",
       "      <td>65.66</td>\n",
       "      <td>74.54</td>\n",
       "      <td>3.57</td>\n",
       "      <td>0.95</td>\n",
       "      <td>57.98</td>\n",
       "      <td>hungarian</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>69.82</td>\n",
       "      <td>29.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1009</td>\n",
       "      <td>28.65</td>\n",
       "      <td>34.91</td>\n",
       "      <td>0.40</td>\n",
       "      <td>1.55</td>\n",
       "      <td>51.04</td>\n",
       "      <td>hungarian</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>31.47</td>\n",
       "      <td>-5.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1009</td>\n",
       "      <td>37.13</td>\n",
       "      <td>45.68</td>\n",
       "      <td>0.40</td>\n",
       "      <td>1.53</td>\n",
       "      <td>49.75</td>\n",
       "      <td>hungarian</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>40.97</td>\n",
       "      <td>3.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>1009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1009</td>\n",
       "      <td>59.40</td>\n",
       "      <td>74.22</td>\n",
       "      <td>1.39</td>\n",
       "      <td>0.90</td>\n",
       "      <td>59.46</td>\n",
       "      <td>hungarian</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>65.99</td>\n",
       "      <td>26.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2287</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2287</td>\n",
       "      <td>50.01</td>\n",
       "      <td>67.33</td>\n",
       "      <td>1.18</td>\n",
       "      <td>1.64</td>\n",
       "      <td>32.93</td>\n",
       "      <td>korean</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>57.40</td>\n",
       "      <td>24.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2287</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2287</td>\n",
       "      <td>27.79</td>\n",
       "      <td>57.57</td>\n",
       "      <td>0.44</td>\n",
       "      <td>1.12</td>\n",
       "      <td>50.90</td>\n",
       "      <td>korean</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>37.49</td>\n",
       "      <td>3.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2287</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2287</td>\n",
       "      <td>33.06</td>\n",
       "      <td>57.19</td>\n",
       "      <td>0.48</td>\n",
       "      <td>1.58</td>\n",
       "      <td>38.78</td>\n",
       "      <td>korean</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>41.90</td>\n",
       "      <td>8.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>2287</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2287</td>\n",
       "      <td>43.02</td>\n",
       "      <td>69.57</td>\n",
       "      <td>0.70</td>\n",
       "      <td>1.18</td>\n",
       "      <td>42.15</td>\n",
       "      <td>korean</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>53.17</td>\n",
       "      <td>20.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>822</td>\n",
       "      <td>75.85</td>\n",
       "      <td>78.20</td>\n",
       "      <td>11.68</td>\n",
       "      <td>2.45</td>\n",
       "      <td>47.45</td>\n",
       "      <td>polish</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>77.00</td>\n",
       "      <td>34.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>822</td>\n",
       "      <td>33.12</td>\n",
       "      <td>34.74</td>\n",
       "      <td>1.22</td>\n",
       "      <td>3.38</td>\n",
       "      <td>33.70</td>\n",
       "      <td>polish</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>33.91</td>\n",
       "      <td>-6.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>822</td>\n",
       "      <td>41.67</td>\n",
       "      <td>44.12</td>\n",
       "      <td>1.95</td>\n",
       "      <td>4.09</td>\n",
       "      <td>25.18</td>\n",
       "      <td>polish</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>42.86</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>822</td>\n",
       "      <td>67.01</td>\n",
       "      <td>68.93</td>\n",
       "      <td>1.46</td>\n",
       "      <td>4.18</td>\n",
       "      <td>29.56</td>\n",
       "      <td>polish</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>67.96</td>\n",
       "      <td>26.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>666</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>666</td>\n",
       "      <td>53.27</td>\n",
       "      <td>58.92</td>\n",
       "      <td>7.81</td>\n",
       "      <td>3.02</td>\n",
       "      <td>32.58</td>\n",
       "      <td>swedish</td>\n",
       "      <td>bert-base-multilingual-cased</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>55.95</td>\n",
       "      <td>26.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>666</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>666</td>\n",
       "      <td>19.29</td>\n",
       "      <td>27.58</td>\n",
       "      <td>0.30</td>\n",
       "      <td>4.74</td>\n",
       "      <td>26.88</td>\n",
       "      <td>swedish</td>\n",
       "      <td>google/canine-c</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>22.70</td>\n",
       "      <td>-2.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>666</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>666</td>\n",
       "      <td>26.80</td>\n",
       "      <td>30.99</td>\n",
       "      <td>0.60</td>\n",
       "      <td>5.22</td>\n",
       "      <td>15.92</td>\n",
       "      <td>swedish</td>\n",
       "      <td>google/canine-s</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>28.74</td>\n",
       "      <td>3.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>not_finetuned</td>\n",
       "      <td>const</td>\n",
       "      <td>single</td>\n",
       "      <td>666</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>666</td>\n",
       "      <td>39.01</td>\n",
       "      <td>46.09</td>\n",
       "      <td>3.00</td>\n",
       "      <td>4.50</td>\n",
       "      <td>24.02</td>\n",
       "      <td>swedish</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>pretrained</td>\n",
       "      <td>42.26</td>\n",
       "      <td>11.59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         finetuned encoding    task  Numberofsentence  NumberofErrorsentence  \\\n",
       "5    not_finetuned    const  single               946                      0   \n",
       "39   not_finetuned    const  single               946                      0   \n",
       "37   not_finetuned    const  single               946                      0   \n",
       "7    not_finetuned    const  single               946                      0   \n",
       "73   not_finetuned    const  single               348                      0   \n",
       "77   not_finetuned    const  single               348                      0   \n",
       "75   not_finetuned    const  single               348                      0   \n",
       "120  not_finetuned    const  single               348                      0   \n",
       "29   not_finetuned    const  single              2416                      0   \n",
       "63   not_finetuned    const  single              2416                      0   \n",
       "61   not_finetuned    const  single              2416                      0   \n",
       "31   not_finetuned    const  single              2416                      0   \n",
       "0    not_finetuned    const  single              2541                      0   \n",
       "33   not_finetuned    const  single              2541                      0   \n",
       "35   not_finetuned    const  single              2541                      0   \n",
       "3    not_finetuned    const  single              2541                      0   \n",
       "65   not_finetuned    const  single              5000                      0   \n",
       "69   not_finetuned    const  single              5000                      0   \n",
       "71   not_finetuned    const  single              5000                      0   \n",
       "67   not_finetuned    const  single              5000                      0   \n",
       "9    not_finetuned    const  single               716                      0   \n",
       "43   not_finetuned    const  single               716                      0   \n",
       "41   not_finetuned    const  single               716                      0   \n",
       "11   not_finetuned    const  single               716                      0   \n",
       "13   not_finetuned    const  single              1009                      0   \n",
       "47   not_finetuned    const  single              1009                      0   \n",
       "45   not_finetuned    const  single              1009                      0   \n",
       "15   not_finetuned    const  single              1009                      0   \n",
       "17   not_finetuned    const  single              2287                      0   \n",
       "51   not_finetuned    const  single              2287                      0   \n",
       "49   not_finetuned    const  single              2287                      0   \n",
       "19   not_finetuned    const  single              2287                      0   \n",
       "21   not_finetuned    const  single               822                      0   \n",
       "55   not_finetuned    const  single               822                      0   \n",
       "53   not_finetuned    const  single               822                      0   \n",
       "23   not_finetuned    const  single               822                      0   \n",
       "25   not_finetuned    const  single               666                      0   \n",
       "59   not_finetuned    const  single               666                      0   \n",
       "57   not_finetuned    const  single               666                      0   \n",
       "27   not_finetuned    const  single               666                      0   \n",
       "\n",
       "     NumberofSkipsentence  NumberofValidsentence  BracketingRecall  \\\n",
       "5                       0                    946             51.29   \n",
       "39                      0                    946             27.73   \n",
       "37                      0                    946             34.92   \n",
       "7                       0                    946             36.10   \n",
       "73                      0                    348             51.85   \n",
       "77                      0                    348             20.57   \n",
       "75                      0                    348             34.74   \n",
       "120                     0                    348             33.63   \n",
       "29                      0                   2416             54.93   \n",
       "63                      0                   2416              8.93   \n",
       "61                      0                   2416             16.54   \n",
       "31                      0                   2416             38.27   \n",
       "0                       0                   2541             49.30   \n",
       "33                      0                   2541             12.22   \n",
       "35                      0                   2541             20.20   \n",
       "3                       0                   2541             31.41   \n",
       "65                      0                   5000             41.19   \n",
       "69                      0                   5000             14.15   \n",
       "71                      0                   5000             21.24   \n",
       "67                      0                   5000             35.89   \n",
       "9                       0                    716             72.48   \n",
       "43                      0                    716             28.71   \n",
       "41                      0                    716             38.46   \n",
       "11                      0                    716             58.61   \n",
       "13                      0                   1009             65.66   \n",
       "47                      0                   1009             28.65   \n",
       "45                      0                   1009             37.13   \n",
       "15                      0                   1009             59.40   \n",
       "17                      0                   2287             50.01   \n",
       "51                      0                   2287             27.79   \n",
       "49                      0                   2287             33.06   \n",
       "19                      0                   2287             43.02   \n",
       "21                      0                    822             75.85   \n",
       "55                      0                    822             33.12   \n",
       "53                      0                    822             41.67   \n",
       "23                      0                    822             67.01   \n",
       "25                      0                    666             53.27   \n",
       "59                      0                    666             19.29   \n",
       "57                      0                    666             26.80   \n",
       "27                      0                    666             39.01   \n",
       "\n",
       "     BracketingPrecision  Completematch  Averagecrossing  Nocrossing  \\\n",
       "5                  63.08           2.33             1.33       46.19   \n",
       "39                 40.84           0.63             2.43       29.28   \n",
       "37                 50.94           0.63             1.66       40.27   \n",
       "7                  67.83           0.32             0.63       68.92   \n",
       "73                 62.35          17.24             3.74       31.90   \n",
       "77                 33.70          16.67             5.97       29.02   \n",
       "75                 45.20          15.52             5.87       25.00   \n",
       "120                46.82          16.95             5.43       26.72   \n",
       "29                 59.77           5.13             4.52       22.68   \n",
       "63                 10.43           1.32            11.43        4.43   \n",
       "61                 18.52           0.75            10.40        5.50   \n",
       "31                 43.06           2.19             7.14        9.19   \n",
       "0                  50.85           3.74             6.28       20.19   \n",
       "33                 12.01           1.50            14.93        9.64   \n",
       "35                 20.07           0.55            12.31        9.13   \n",
       "3                  33.33           1.34             9.80       12.91   \n",
       "65                 50.54          10.84             1.11       50.36   \n",
       "69                 26.54           5.10             1.35       45.98   \n",
       "71                 27.55           5.72             1.90       34.12   \n",
       "67                 48.21           8.00             1.09       50.38   \n",
       "9                  76.53           1.12             5.70       18.58   \n",
       "43                 31.02           0.00            13.78        4.19   \n",
       "41                 41.99           0.14            10.78        6.15   \n",
       "11                 61.32           0.00            10.50        5.45   \n",
       "13                 74.54           3.57             0.95       57.98   \n",
       "47                 34.91           0.40             1.55       51.04   \n",
       "45                 45.68           0.40             1.53       49.75   \n",
       "15                 74.22           1.39             0.90       59.46   \n",
       "17                 67.33           1.18             1.64       32.93   \n",
       "51                 57.57           0.44             1.12       50.90   \n",
       "49                 57.19           0.48             1.58       38.78   \n",
       "19                 69.57           0.70             1.18       42.15   \n",
       "21                 78.20          11.68             2.45       47.45   \n",
       "55                 34.74           1.22             3.38       33.70   \n",
       "53                 44.12           1.95             4.09       25.18   \n",
       "23                 68.93           1.46             4.18       29.56   \n",
       "25                 58.92           7.81             3.02       32.58   \n",
       "59                 27.58           0.30             4.74       26.88   \n",
       "57                 30.99           0.60             5.22       15.92   \n",
       "27                 46.09           3.00             4.50       24.02   \n",
       "\n",
       "      Treebank                Language Model  Pretrained  F-Score  Difference  \n",
       "5       basque  bert-base-multilingual-cased  pretrained    56.58       24.13  \n",
       "39      basque               google/canine-c  pretrained    33.03       -3.13  \n",
       "37      basque               google/canine-s  pretrained    41.43        5.27  \n",
       "7       basque              xlm-roberta-base  pretrained    47.12       13.44  \n",
       "73     chinese  bert-base-multilingual-cased  pretrained    56.61       40.47  \n",
       "77     chinese               google/canine-c  pretrained    25.55        8.70  \n",
       "75     chinese               google/canine-s  pretrained    39.28       21.94  \n",
       "120    chinese              xlm-roberta-base  pretrained    45.60       37.40  \n",
       "29     english  bert-base-multilingual-cased  pretrained    57.25       44.36  \n",
       "63     english               google/canine-c  pretrained     9.62       -0.27  \n",
       "61     english               google/canine-s  pretrained    17.48        7.59  \n",
       "31     english              xlm-roberta-base  pretrained    40.52       26.06  \n",
       "0       french  bert-base-multilingual-cased  pretrained    50.06       35.57  \n",
       "33      french               google/canine-c  pretrained    12.11       -1.98  \n",
       "35      french               google/canine-s  pretrained    20.14        6.27  \n",
       "3       french              xlm-roberta-base  pretrained    32.35       16.98  \n",
       "65      german  bert-base-multilingual-cased  pretrained    45.39       26.64  \n",
       "69      german               google/canine-c  pretrained    18.46        3.08  \n",
       "71      german               google/canine-s  pretrained    23.98        7.61  \n",
       "67      german              xlm-roberta-base  pretrained    41.15       23.03  \n",
       "9       hebrew  bert-base-multilingual-cased  pretrained    74.45       32.93  \n",
       "43      hebrew               google/canine-c  pretrained    29.82      -11.01  \n",
       "41      hebrew               google/canine-s  pretrained    40.15       -0.81  \n",
       "11      hebrew              xlm-roberta-base  pretrained    59.93       16.64  \n",
       "13   hungarian  bert-base-multilingual-cased  pretrained    69.82       29.79  \n",
       "47   hungarian               google/canine-c  pretrained    31.47       -5.90  \n",
       "45   hungarian               google/canine-s  pretrained    40.97        3.60  \n",
       "15   hungarian              xlm-roberta-base  pretrained    65.99       26.44  \n",
       "17      korean  bert-base-multilingual-cased  pretrained    57.40       24.17  \n",
       "51      korean               google/canine-c  pretrained    37.49        3.67  \n",
       "49      korean               google/canine-s  pretrained    41.90        8.08  \n",
       "19      korean              xlm-roberta-base  pretrained    53.17       20.32  \n",
       "21      polish  bert-base-multilingual-cased  pretrained    77.00       34.21  \n",
       "55      polish               google/canine-c  pretrained    33.91       -6.20  \n",
       "53      polish               google/canine-s  pretrained    42.86        2.75  \n",
       "23      polish              xlm-roberta-base  pretrained    67.96       26.17  \n",
       "25     swedish  bert-base-multilingual-cased  pretrained    55.95       26.17  \n",
       "59     swedish               google/canine-c  pretrained    22.70       -2.80  \n",
       "57     swedish               google/canine-s  pretrained    28.74        3.24  \n",
       "27     swedish              xlm-roberta-base  pretrained    42.26       11.59  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Language: UD_Ancient_Greek-Perseus, Trees: 13919, Tokens: 216911, Unique Tokens: 41563\n",
      "Language: UD_Skolt_Sami-Giellagas, Trees: 200, Tokens: 2664, Unique Tokens: 738\n",
      "Language: UD_Welsh-CCG, Trees: 2111, Tokens: 43322, Unique Tokens: 6936\n",
      "Language: UD_Bulgarian-BTB, Trees: 11138, Tokens: 167290, Unique Tokens: 28802\n",
      "Language: UD_Guajajara-TuDeT, Trees: 284, Tokens: 2339, Unique Tokens: 480\n",
      "Language: UD_Armenian-ArmTDP, Trees: 2502, Tokens: 55135, Unique Tokens: 14120\n",
      "Language: UD_Turkish-BOUN, Trees: 9761, Tokens: 132147, Unique Tokens: 38016\n",
      "Language: UD_Ligurian-GLT, Trees: 316, Tokens: 7247, Unique Tokens: 1666\n",
      "Language: UD_Vietnamese-VTB, Trees: 3000, Tokens: 46757, Unique Tokens: 5818\n",
      "Language: UD_Basque-BDT, Trees: 8993, Tokens: 130439, Unique Tokens: 26680\n",
      "Language: UD_Bhojpuri-BHTB, Trees: 357, Tokens: 7025, Unique Tokens: 1681\n",
      "Language: UD_Kiche-IU, Trees: 1435, Tokens: 11451, Unique Tokens: 2397\n",
      "Language: UD_Chinese-GSDSimp, Trees: 4997, Tokens: 128291, Unique Tokens: 20157\n",
      "Language: UD_Classical_Chinese-Kyoto, Trees: 58301, Tokens: 341470, Unique Tokens: 8467\n",
      "Language: UD_Naija-NSC, Trees: 9241, Tokens: 150043, Unique Tokens: 5060\n",
      "Language: UD_Maltese-MUDT, Trees: 2074, Tokens: 46239, Unique Tokens: 8472\n",
      "Language: UD_Gothic-PROIEL, Trees: 5401, Tokens: 60740, Unique Tokens: 8766\n",
      "Language: UD_Wolof-WTB, Trees: 2107, Tokens: 46368, Unique Tokens: 5957\n",
      "Language: UD_Old_East_Slavic-TOROT, Trees: 16944, Tokens: 166727, Unique Tokens: 32643\n"
     ]
    }
   ],
   "source": [
    "treebanks = [\n",
    "    'UD_Ancient_Greek-Perseus', 'UD_Skolt_Sami-Giellagas', 'UD_Welsh-CCG',\n",
    "    'UD_Bulgarian-BTB', 'UD_Guajajara-TuDeT', 'UD_Armenian-ArmTDP',\n",
    "    'UD_Turkish-BOUN', 'UD_Ligurian-GLT', 'UD_Vietnamese-VTB',\n",
    "    'UD_Basque-BDT', 'UD_Bhojpuri-BHTB', 'UD_Kiche-IU', 'UD_Chinese-GSDSimp',\n",
    "    'UD_Classical_Chinese-Kyoto', 'UD_Naija-NSC', 'UD_Maltese-MUDT',\n",
    "    'UD_Gothic-PROIEL', 'UD_Wolof-WTB', 'UD_Old_East_Slavic-TOROT',\n",
    "        ]\n",
    "\n",
    "trees_dic = {}\n",
    "# measure number of trees and tokens of constituency treebanks, using seq files\n",
    "# define data directory\n",
    "for language in treebanks:\n",
    "    data_dir = 'data/rel-pos/not_finetuned/pretrained/{}/single/'.format(language)\n",
    "    train = data_dir + 'train'\n",
    "    dev = data_dir + 'dev'\n",
    "    test = data_dir + 'test'\n",
    "    tokens_unique = set()\n",
    "    tokens = []\n",
    "    trees_ = []\n",
    "    with open(train, 'r') as f:\n",
    "        text = f.read()\n",
    "        # get number of trees\n",
    "        trees = len(text.split('\\n\\n')) - 1\n",
    "        # get number of tokens\n",
    "        lines = text.split('\\n')\n",
    "        for line in lines:\n",
    "            line = line.split('\\t')\n",
    "            try:\n",
    "                if line[0] not in ('-BOS-', '-EOS-'):\n",
    "                    tokens_unique.add(line[0])\n",
    "                    tokens.append(line[0])\n",
    "            except IndexError:\n",
    "                continue\n",
    "        trees_.append(trees)\n",
    "    \n",
    "    with open(dev, 'r') as f:\n",
    "        text = f.read()\n",
    "        # get number of trees\n",
    "        trees = len(text.split('\\n\\n')) - 1\n",
    "        # get number of tokens\n",
    "        lines = text.split('\\n')\n",
    "        for line in lines:\n",
    "            line = line.split('\\t')\n",
    "            try:\n",
    "                if line[0] not in ('-BOS-', '-EOS-'):\n",
    "                    tokens_unique.add(line[0])\n",
    "                    tokens.append(line[0])\n",
    "            except IndexError:\n",
    "                continue\n",
    "        trees_.append(trees)\n",
    "    \n",
    "    with open(test, 'r') as f:\n",
    "        text = f.read()\n",
    "        # get number of trees\n",
    "        trees = len(text.split('\\n\\n')) - 1\n",
    "        # get number of tokens\n",
    "        lines = text.split('\\n')\n",
    "        for line in lines:\n",
    "            line = line.split('\\t')\n",
    "            try:\n",
    "                if line[0] not in ('-BOS-', '-EOS-'):\n",
    "                    tokens_unique.add(line[0])\n",
    "                    tokens.append(line[0])\n",
    "            except IndexError:\n",
    "                continue\n",
    "        trees_.append(trees)\n",
    "    \n",
    "    trees = sum(trees_)\n",
    "    tokens = len(tokens)\n",
    "    tokens_unique = len(tokens_unique)\n",
    "    print('Language: {}, Trees: {}, Tokens: {}, Unique Tokens: {}'.format(language, trees, tokens, tokens_unique)) \n",
    "    trees_dic[language] = [trees, tokens, tokens_unique]     \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trees</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Unique Tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UD_Skolt_Sami-Giellagas</th>\n",
       "      <td>200</td>\n",
       "      <td>2664</td>\n",
       "      <td>738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Guajajara-TuDeT</th>\n",
       "      <td>284</td>\n",
       "      <td>2339</td>\n",
       "      <td>480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Ligurian-GLT</th>\n",
       "      <td>316</td>\n",
       "      <td>7247</td>\n",
       "      <td>1666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Bhojpuri-BHTB</th>\n",
       "      <td>357</td>\n",
       "      <td>7025</td>\n",
       "      <td>1681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Kiche-IU</th>\n",
       "      <td>1435</td>\n",
       "      <td>11451</td>\n",
       "      <td>2397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Maltese-MUDT</th>\n",
       "      <td>2074</td>\n",
       "      <td>46239</td>\n",
       "      <td>8472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Wolof-WTB</th>\n",
       "      <td>2107</td>\n",
       "      <td>46368</td>\n",
       "      <td>5957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Welsh-CCG</th>\n",
       "      <td>2111</td>\n",
       "      <td>43322</td>\n",
       "      <td>6936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Armenian-ArmTDP</th>\n",
       "      <td>2502</td>\n",
       "      <td>55135</td>\n",
       "      <td>14120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Vietnamese-VTB</th>\n",
       "      <td>3000</td>\n",
       "      <td>46757</td>\n",
       "      <td>5818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Chinese-GSDSimp</th>\n",
       "      <td>4997</td>\n",
       "      <td>128291</td>\n",
       "      <td>20157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Gothic-PROIEL</th>\n",
       "      <td>5401</td>\n",
       "      <td>60740</td>\n",
       "      <td>8766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Basque-BDT</th>\n",
       "      <td>8993</td>\n",
       "      <td>130439</td>\n",
       "      <td>26680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Naija-NSC</th>\n",
       "      <td>9241</td>\n",
       "      <td>150043</td>\n",
       "      <td>5060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Turkish-BOUN</th>\n",
       "      <td>9761</td>\n",
       "      <td>132147</td>\n",
       "      <td>38016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Bulgarian-BTB</th>\n",
       "      <td>11138</td>\n",
       "      <td>167290</td>\n",
       "      <td>28802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Ancient_Greek-Perseus</th>\n",
       "      <td>13919</td>\n",
       "      <td>216911</td>\n",
       "      <td>41563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Old_East_Slavic-TOROT</th>\n",
       "      <td>16944</td>\n",
       "      <td>166727</td>\n",
       "      <td>32643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UD_Classical_Chinese-Kyoto</th>\n",
       "      <td>58301</td>\n",
       "      <td>341470</td>\n",
       "      <td>8467</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Trees  Tokens  Unique Tokens\n",
       "UD_Skolt_Sami-Giellagas       200    2664            738\n",
       "UD_Guajajara-TuDeT            284    2339            480\n",
       "UD_Ligurian-GLT               316    7247           1666\n",
       "UD_Bhojpuri-BHTB              357    7025           1681\n",
       "UD_Kiche-IU                  1435   11451           2397\n",
       "UD_Maltese-MUDT              2074   46239           8472\n",
       "UD_Wolof-WTB                 2107   46368           5957\n",
       "UD_Welsh-CCG                 2111   43322           6936\n",
       "UD_Armenian-ArmTDP           2502   55135          14120\n",
       "UD_Vietnamese-VTB            3000   46757           5818\n",
       "UD_Chinese-GSDSimp           4997  128291          20157\n",
       "UD_Gothic-PROIEL             5401   60740           8766\n",
       "UD_Basque-BDT                8993  130439          26680\n",
       "UD_Naija-NSC                 9241  150043           5060\n",
       "UD_Turkish-BOUN              9761  132147          38016\n",
       "UD_Bulgarian-BTB            11138  167290          28802\n",
       "UD_Ancient_Greek-Perseus    13919  216911          41563\n",
       "UD_Old_East_Slavic-TOROT    16944  166727          32643\n",
       "UD_Classical_Chinese-Kyoto  58301  341470           8467"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(trees_dic, orient='index', columns=['Trees', 'Tokens', 'Unique Tokens']).sort_values(by='Trees')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml_probing",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13 (default, Mar 29 2022, 02:18:16) \n[GCC 7.5.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "085035040a5f8541f0a11a144df6817fe51afa0926562ddfedc7359bee17119a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
